# -*- coding: utf-8 -*-
"""
@Time    : 2021/10/11 19:22
@Author  : moqilintong

迭代地解析网页，并把需要的数据放到输出对象中
"""
import html
import re

import requests
import urllib3
from bs4 import BeautifulSoup

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36 Edg/91.0.864.59'
}
# urllib3.disable_warnings()


def parse_html(url: str, output: list):
    while True:
        # 爬取网页
        response = requests.get(url, headers=headers, verify=False)
        response.encoding = 'utf-8'
        html_text = response.text

        # 30-38行 分析网页结构，获取需要的学校名称、联系电话、邮编和地址等信息
        # 41-44行 将获取到的数据放入到 primaryschool 对象中
        bs = BeautifulSoup(html_text, 'lxml')
        target = bs.find(attrs={'class': 'main'})
        sk_list = target.findAll(attrs={'class': 'sk'})
        for sk in sk_list:
            sk_text = str(sk)
            school = re.findall(r'target="_blank">([^<]*)(?=</a></h4>)', sk_text, re.DOTALL)
            telephone = re.findall(r'电话：([^<]*)(?=<br)', sk_text, re.DOTALL)
            postcode = re.findall(r'邮编：([^<]*)(?=<br)', sk_text, re.DOTALL)
            address = re.findall(r'地址：([^<]*)(?=<div)', sk_text, re.DOTALL)

            # 学校 联系电话 邮编 地址
            struct = [school[0], address[0]]
            struct.insert(1, telephone[0] if telephone else None)
            struct.insert(2, postcode[0] if postcode else None)
            output.append(struct)

        # 当前网页是否存在下一页
        # 如果存在，迭代解析网页
        # 如果不存在，停止迭代，将 输出对象output 返回上一级程序
        prepare = target.find(attrs={'class': 'fy'})
        next_page_link = re.findall(r'<a href="([^<]*)">(?=下一页)', str(prepare), re.DOTALL)
        if next_page_link:
            url = html.unescape(next_page_link[0])
        else:
            break
